# -*- coding: UTF-8 -*-
# cython: language_level=3
import re
import socket

import requests
from lxml import etree
from requests.utils import get_encodings_from_content

from db.mongodb import MongoDB
from db.oracle_db import OracleDB
from db.proxy_db import ProxyDB
from utils.ip_area import find_ip
from utils.own_tools import get_host

oracledb = OracleDB()
mongodb = MongoDB()


def fix_icp():
    add_sql = "SELECT NAME,URL,to_char(RECORD_DATE, 'SYYYY-MM-DD HH24:MI:SS') FROM TAB_MANAGE_SEARCH_INFO " \
              "WHERE CRACK_STATUS = 0 AND SEARCH_TYPE = 3"
    items = oracledb.find(add_sql)
    for title, item_url, RECORD_DATE in items:
        try:
            RECORD_DATE_mongo = RECORD_DATE or ''
            RECORD_DATE = f"TO_DATE('{RECORD_DATE}', 'YYYY-MM-DD HH24:MI:SS')" if RECORD_DATE else 'NULL'
            try:
                proxy = ProxyDB().get_proxy()
                proxy = {'http': proxy, 'https': proxy} if proxy else None
                response = requests.get(item_url, proxies=proxy, stream=True)
            except:
                continue
            try:
                ip = response.raw._connection.sock.getpeername()[0]
            except:
                print('request erro')
                ip = socket.getaddrinfo(item_url.split('://')[-1].split('/')[0], None)[0][4][0]
            try:
                country, province_name, city_name, isp = find_ip(ip)
                if country and country != '中国':
                    province_name = country + province_name

            except:
                province_name, city_name = '', ''

            if response.encoding == 'ISO-8859-1':
                encodings = get_encodings_from_content(response.text)
                encodings = encodings[0] if encodings else response.apparent_encoding

                if (not encodings) or (encodings.lower() not in ['gbk', 'utf-8', 'utf8', 'gb2312']):
                    response.encoding = 'gbk'
                else:
                    response.encoding = encodings

            htmlstr = response.text
            html = etree.HTML(htmlstr)
            title_auto = html.xpath('//title/text()')
            title_auto = title_auto[0] if title_auto else ''

            icp = re.search('([\u4e00-\u9fa5]\s*ICP\s*[\u4e00-\u9fa5]*\s*\d{4,12}\s*号*-*\d*)', htmlstr)
            icp = icp.group(1) if icp else ''

            print(icp)
            video_license = re.search(
                '((信息网络传播视听节目许可证|信息网络传播视听许可证|视听许可证).{0,4}\d{0,12}号*)', htmlstr)
            video_license = video_license.group(1) if video_license else ''

            title = title if title else title_auto
            title = 'NULL' if not title else f"'{title}'"

            # {'host': 'weibo.com', 'home_page': 'https://aaa.weibo.com'}
            get_host_r = get_host(item_url)
            host = get_host_r['host']
            home_page = get_host_r['home_page']

            # 补全oracle数据
            sql = "INSERT INTO TAB_VIDEO_SITE_INFO (ID, SITE_ID,URL,AUDIO_LICENSE,IP_ADDRESS,CITY_NAME,PROVINCE_NAME," \
                  "DOMAIN,FIND_DATE,SITE_NAME,AUTO_RECORD_NUMBER,SITE_TYPE,ADD_TYPE) " \
                  "values(SEQ_NO_LICENSE_VIDEO.nextval, SEQ_NO_LICENSE_VIDEO.nextval," \
                  f"'{home_page}','{video_license}','{ip}','{city_name}','{province_name}','{host}',{RECORD_DATE}," \
                  f"{title},'{icp}',3,2 )"
            if oracledb.add(sql):
                item_id = f"SELECT ID FROM TAB_VIDEO_SITE_INFO WHERE DOMAIN = '{host}'"
                item_id = oracledb.find(item_id)[0][0]
                mongodb.delete('WEB_urls', {'host': host})
                mongodb.add('WEB_urls',
                            {"oid": item_id, 'host': host, "home_page": home_page,
                             'record_time': RECORD_DATE_mongo, 'title': title, "OracleStatus": True})
            sql = f"UPDATE TAB_MANAGE_SEARCH_INFO SET CRACK_STATUS = 2 WHERE URL='{item_url}'"
            oracledb.update(sql)
        except:
            pass


if __name__ == '__main__':
    fix_icp()
